{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The config dir path: /Users/xuanyidong/Desktop/AutoDL-Projects/configs\n"
     ]
    }
   ],
   "source": [
    "#####################################################\n",
    "# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.06 #\n",
    "#####################################################\n",
    "import os, sys, math\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "from xautodl.xmisc.scheduler_utils import CosineParamScheduler, MultiStepParamScheduler\n",
    "from xautodl.xmisc.scheduler_utils import LRMultiplier, WarmupParamScheduler\n",
    "\n",
    "__file__ = os.path.dirname(os.path.realpath(\"__file__\"))\n",
    "\n",
    "config_dir = (Path(__file__).parent / \"..\" / \"configs\").resolve()\n",
    "print(\"The config dir path: {:}\".format(config_dir))\n",
    "\n",
    "def draw(steps, lrs):\n",
    "    plt.close()\n",
    "    dpi, width, height = 200, 1400, 800\n",
    "    figsize = width / float(dpi), height / float(dpi)\n",
    "    fig = plt.figure(figsize=figsize)\n",
    "    ax = fig.add_subplot(111)\n",
    "\n",
    "    plt.plot(steps, lrs)\n",
    "    plt.title(\"Plot Cosine Decayed LR with Warmup\")\n",
    "    plt.xlabel(\"steps\")\n",
    "    plt.ylabel(\"learning rate\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAEWCAYAAAD7HukTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABBaUlEQVR4nO3dd3hUZdrH8e+dShIg9N577yBgryCK2Luuuq6rq7vruq762nvX1bUhuuraxY4KKhaKYqH3FkILvQbIhExmcr9/nBMdYxIGyORMuT/XNVdmzpyZ+c3JZO485zzneURVMcYYY+JRktcBjDHGmEixImeMMSZuWZEzxhgTt6zIGWOMiVtW5IwxxsQtK3LGGGPilhU5UyERmSQil3udozIicrOIvOh1jmglIqtE5Divc5QSkT0i0q6S+6Mqr4l9VuQSnPulUuh++WwSkZdFpOZ+PkcbEVERSdnHep1E5F0R2Soi+SIyT0SuE5HkA82vqverapUXYhE5SkRK3O2yR0TyRGSsiAys6tfyioi8IiL3VnCfikiB+97XicjjB/N7KqWqNVU1d1+vvy8ikuJmGxSy7AI3d9llSw42t4ldVuQMwEhVrQn0AwYCt1b1C4hIe+AnYC3QU1WzgbOAAUCtqn69KrLe3S61gMHAEmCqiBzrbaxq09t9/0cC5wCXeZznF6oaAH7AyVbqCJzfUdllU/bnucVh341xwn6R5hequg6YAPQoe5+IJInIrSKyWkQ2i8irIpLt3l36JbLT/e96SDlPfxcwTVWvU9UN7ustVdXzVXWn+xqniMhCEdnp7irtGvL6N7otit0isrS00IjInSLyunu9tEX5BxFZ47YYbynzHm4SkRUiss1tmdULY7uoquap6u3Ai8BDIc/ZRUQmish2N9fZIfdliMhj7jbLF5HvRCTDve9dEdnoLp8iIt3d5QPdFnVKyPOcISJzwnkPInKR+3rbQt/7wVDVHOB7oE9594vIpSLyScjtHBEZG3J7rYj0ca+riHQQkSuAC4Ab3M/MJyFP2cdt5eeLyDsiUqOCaFNwilipw3F+N2WXTRGRuiLyqYhsEZEd7vUWIRknich9IvI94APauVn/IiLL3c/dPSLSXkR+EJFd7rZPcx9/iYh8V2a7qIh0cK+/IiKj3c/KbhGZLCKtK3hfpgpZkTO/EJGWwAhgdjl3X+JejgbaATWBp937Sr9U6ri7o34o5/HHAe9V8tqdgLeAa4GGwHjgExFJE5HOwDXAQFWtBQwDVlXyVg4DOgPHAreHFMu/Aafi/KffDNgBPFPJ85TnA6CfiGSJSBYwEXgTaAScBzxbWrCAR4H+wFCgHnADUOLeNwHo6D5uFvAGgKpOB7YBx4e85oXAa/t6DyLSDXgOuMi9rz7QgoMkIl1wikVOBatMBg53C3BTIBU41H1s6WdlXugDVHUMznt+2P3MjAy5+2xgONAW6IXzuSvPFOBQ93UbAFnAWGBQyLIu7npJwMtAa6AVUMivn99SFwFX4LTcV7vLhuP8Dgfj/P7G4BTnljj/DJ5XQbbyXADcAzQA5rjv30SaqtolgS84xWIPsBPnD/tZIMO9bxJwuXv9a+AvIY/rDBQDKUAbQIGUSl6nGBheyf23AWNDbicB64CjgA7AZpxCmVrmcXcCr7vXS3O0CLn/Z+Bc9/pi4NiQ+5qWvody8hwF5JWzvIv7Gs1xduFNLXP/88Adbv5CnF1++/od1HGfM9u9fSPwhnu9Hk7Loum+3gNwO/B2yH1ZgB84roLXfQW4t4L7FNgFFLjX3wLSK3kPa3F2d5+LUwh+drfVpcC4Ms/boaLXdz+PF4bcfhgYXcFr1gD2Ar2B00K22Y8hy1ZW8Ng+wI6Q25OAu8vZBoeG3J4J3Bhy+zHgCff6JcB35Tw+9L2G/m5qAkGgZVX/Tdvlt5dKOwqYhHGqqn61j3Wa8et/t7jXU4DGYb7GNpwv5LCeX1VLRGQt0FxVJ4nItTgFrbuIfAFcp6rrK3iujSHXfThfKOD8F/+hiJSE3B9038O6MN9Hc5wvr53u8x0iIjtD7k/BaXU1wPkSXlH2CcTpwHEfzjHJhvzaumsA5AOvA4vF6QB0Nk4h3RDGe2iGU2wAUNUCEdkW5vsqTz83/1nAgzhFs6iCdSfz6z8kk3G2z5HAEPf2/ij7+2tW3kqquldEfsbZk9AOmOre9V3IsikAIpIJ/BunZVbXXa+WiCSratC9/cu2C7Ep5HphObebhPmefvP8qrpHRLZT5ndmqp7trjThWo/zBVuqFRDA+aMPZyqLr4Azwn1+ERGcXULrAFT1TVU9zF1HCTkuth/WAieqap2QSw11jkWG6zRglqoWuM83uczz1VTVq4CtOK2M9uU8x/nAKJyWaTZOCxRA4Jdjoz+4r3URv+6q3Nd72ICzzZwnc77Y6+/He/sddYx189xeyaqlRe5w9/pknCJ3JBUXuaqYAqX0uNzh/FrkpoYsKz1e/E+cvQ+HqGptft3FLlWUpwDILL0hIuUVv9DfTU2cVnpF/6iZKmJFzoTrLeAfItLW/QO9H3hHnV5uW3BaIxWe/4SzC2+oiDxS+gXgdkB4XUTq4BxLOUlEjhWRVJwvpSJgmoh0FpFjRCQdp3AU4rRe9tdo4L7SA/4i0lBERu3rQeJoLiJ3AJcDN7t3fQp0cjt7pLqXgSLSVVVLgJeAx0WkmYgki8gQ9z3Uct/bNpwvxvvLedlXcY4B9QQ+DPM9vAecLCKHuR0i7mbff+PJIlIj5JJWwXoPAldU8OUNTiE7GmdXdx5OoRmOU2TLO8YLzj9IlX1mwjHFfd2WwCJ32Xc4BbcPvxa5Wjifm53idNS54yBft6y5OHsZ+rgdZe4sZ50RIb+be4CfVNVacRFmRc6E6yWcFsUUYCVOsfkrgKr6cHa/fS9Oz8jBZR+sqitwdl21ARaKSD7wPjAD2K2qS3E6WDyF0woaiXNqgx9Ix/mS3YqzK6sRvxaa/fEkMA74UkR24xy7OaSS9ZuJyB6cY5bTcQrOUar6pfuedgMn4ByHWu9me8jNC3A9MN997Hb3viScArYap5W6yM1R1oe4uybdVuM+34OqLgSuxukIswGnU0rePrbJTThf/qWXb8pbSVXn4xSyf1Vw/zKc7TTVvb0LyAW+D9kdWNZ/gW7uZ+ajfeSsyDSc1vBPqu6BMNVtOP94bVbV5e56TwAZOJ+hH4HPD/D1yuW+/7tx9lgsxym0Zb2JU1y343RmuaAqM5jyifu5MMZEGRFZAfw5jOOlJsqJyCs4HZmq/BxUUzlryRkThUTkDJxjROW2rIwx4bHelcZEGRGZBHQDLnKP7RljDpDtrjTGGBO3bHelMcaYuBVzuysbNGigbdq08TqGMcaYKDJz5sytqtqw7PKYK3Jt2rRhxowZXscwxhgTRURkdXnLbXelMcaYuGVFzhhjTNyyImeMMSZuWZEzxhgTt6zIGWOMiVsRLXIiMlxElopIjojcVM79R4kzxf0c91LZVB7GGGPMfonYKQTuxJDPAMfjjIQ+XUTGqeqiMqtOVdWTI5XDGGNM4orkeXKDgBxVzQUQkbdxJoosW+SMMcYkAFVl3c5CNuTvZUP+XjbmF5KZlsKFg1vv+8EHKJJFrjm/ndY9j/Ln7hoiInNx5uO63p0T6zdE5ArgCoBWrVpFIOrv5W7ZQ3FQ6dS4Js4k1cYYY/ZHsESZsWo7P6/czuy1O5m9Zgc7fMW/Wad3i+yYLXLlVYayo0HPAlqr6h4RGQF8BHT83YNUxwBjAAYMGFAtI0pf/uoMcrcU0K5BFsN7NGFEz6b0aJ5dHS9tjDExK1iiTFuxlQkLNvLlwo1s3eNHBDo0rMnx3RrTq0UdWtXLpGl2DZpk16BWjdSI5olkkcvDmZK+VAuc1tov3NmDS6+PF5FnRaSBqm6NYK6wbC/w07tlHWrXSOH5Kbk8O2kFvVvW4dKhbRjRsylpKdYx1RhjSuUXFjN2+lr+98Mq8nYUkpmWzNFdGjGiR1MO69iA7IzIFrOKRLLITQc6ikhbYB1wLnB+6Aoi0gTYpKoqIoNwentui2CmsPn8QYa0q89NJ3ZhR4GfT+at55Vpq7j2nTncP34xlx7alkuGtiEjLdnrqMYY45nNu/byzLc5vDszD58/yKC29fi/E7tybNdG1Ej1/vsxYkVOVQMicg3wBZAMvKSqC0XkSvf+0cCZwFUiEgAKgXM1Cia4Kw6W4A+UkOkWsLpZaVw8pA0XHtKaKcu38N/vVvLQ50t4ZdpK/n5sJ84e0IKUZGvZGWMSx669xYyZnMt/v1tJcbCEUX2ac+mhbaLusE5EZyFQ1fHA+DLLRodcfxp4OpIZDoTPHwT4pciVSkoSjurciKM6N+Lnldt5cMJibv5wPi9+l8udI7tzRKffzfJgjDFxpaREefPnNTz25VJ2+IoZ2bsZ/zy+E20aZHkdrVwxN9VOdSh0i1xWesWbZ1Dberx/1VAmLtrEAxOWcPFLP3NGvxbcelJX6malVVdUY4ypNjmb9/B/H8xj+qodDG5Xj1tGdKNni+hquZVlRa4cBf4A8PuWXFkiwgndm3BEp4Y89c1ynp+cy6Slm7lrVHdO7tWsOqIaY0zEBYIlPDdpBU99k0NGWjIPn9mLs/q3iInTq6zIlaPwl92V4W2eGqnJ/GtYF07u1Yyb3p/HNW/OZuqyrdx5SnfrmGKMiWkb8gv521uzmb5qByf1bModp3SjUa0aXscKmxW5chQUOS25rP0sUF2b1ub9q4by76+W8eykFcxas4NnLuhHp8a1IhHTGGMi6uvFm/jnu3PxB0r49zm9Oa1vC68j7TfrEliO0o4nB9IKS0lO4l/DuvDqZYPY4fNzytPf8eHsvKqOaIwxERMsUR6YsJg//m8GzbIz+PSvh8VkgQMrcuXyhdHxZF8O79iQ8X8/nN4t6vCPd+byyBdLKCnx/OwIY4yp1J6iAH9+bQbPT87l/ENa8cFfhtKuYU2vYx0wK3LlKO14knGQJzI2qlWD1/54COcObMkz367gL2/Mwuc+tzHGRJu8HT7OfG4a3y7dwt2junP/aT2j4oTug2FFrhy+0mNyB9GSK5WWksQDp/fk1pO68uWijZz9/A9s2V100M9rjDFVacG6fE59Zhrrdhby8iUDuXhIG68jVQkrcuXwFZd/MviBEhEuP7wdL/5hADmb93D28z+wbmdhlTy3McYcrJ9yt3HumB9JT0niw78MjauBLazIlcNXFCRJIL2KB2E+pktjXv/jIWzdU8SZz00jZ/OeKn1+Y4zZX98s2cTFL/1Mk+wavHfVEDo0iq/e4FbkyuHzB8lKS4nIiY4D2tTj7SsGUxws4eznf2DBuvwqfw1jjAnHx3PWccWrM+nUuBZj/zyEptkZXkeqclbkyuHzB8hMj9zB1u7Nsnn3yqFkpCZzwYs/sWj9rn0/yBhjqtC4uev5xztz6N+6Lm/+6RDqxelwhFbkylHgD4Y92smBatsgi7evGExmWjIX/vcnlm7cHdHXM8aYUuPnb+Af78xhYJt6vHLpoIhPXOolK3LlKPQHqqzTSWVa1svkrT8NJjVZuODFH8nZbIXOGBNZXy7cyN/emk3flnV46ZKBcT/0oBW5chQUBaulyAG0aZDFm38aDAjnvfATq7cVVMvrGmMSz+RlW7j6zVn0aJ7Ny5cOrJLTpKKdFbly+PyBiO+uDNW+YU3e+tMhFAdL+MNLP7N1j51HZ4ypWnPX7uSq12fSsVEt/ndZfO+iDGVFrhw+f5CsCHY8KU/HxrV46ZKBbNy1l0tfns6eIhsZxRhTNXK37OHSV6ZTv2Yar1w2kOyMxChwYEWuXD5/kIzU6m/G92tVl2fO78eiDbu46vWZ+AMl1Z7BGBNfNu/ay8Uv/QzAq5cdElPT5FQFK3Ll8PkD1d6SK3Vs18Y8cHpPpi7fyo3vz0PVBnU2xhyYgqIAl7w8ne0Ffl6+ZCBtG2R5Hanaxf9RxwNQHacQVObsAS3ZmL+Xxycuo33DLK45pqNnWYwxsSlYovz97Tks2biL/14ykN4t63gdyRNW5MoIBEvwB0qqrXdlRf56TAdyt+zh0S+X0a5hTUb0bOppHmNMbHn48yV8tXgTd47sxtGdG3kdxzO2u7KMqh6c+UCJCA+e0Yt+repw3dg5zM+z4b+MMeEZO2Mtz0/J5cLBrfjD0DZex/GUFbkyfEWlRc77Rm6N1GSev2gA9bPSufzV6WzatdfrSMaYKPdT7jZu+XA+h3VowB0ju0dkDN5YYkWujNIJU73qeFJWw1rpvPiHAezeG7Ael8aYSm3IL+TqN2fRsm4mz5zfj9Rk+4q3LVBGoT96WnKlujatzcNn9mLWmp3c8+kir+MYY6JQUSDIVa/PotAf5PmL+pOdmTjnwlUmer7Jo0SBexK218fkyjq5VzPm5eUzZkouvVpkc9aAll5HMsZEkbs/WcSctTt59oJ+dGwcX3PCHQxryZURLR1PynPDsM4MaVefWz5aYPPQGWN+MXb6Wt74aQ1/PrKd9cQuw4pcGaUdT6Jx4NKU5CSePr8vDbLSuPL1meQXFnsdyRjjsYXr87n14wUc2qE+/zqhs9dxoo4VuTJKO55kpEZfSw6gfs10nr6gHxvz93KTjYhiTEIrKArw1zdnUycjlf+c25cU62jyO7ZFyijteBKNLblS/VrV5fphnZmwYCNv/LTG6zjGGI/c/vFCVm4r4Ilz+1C/ZrrXcaKSFbkySlty0XhMLtQVh7fjiE4NufvTRSzesMvrOMaYavbBrDzen5XHX4/pyND2DbyOE7UiWuREZLiILBWRHBG5qZL1BopIUETOjGSecPiKgiQJpKdEd/1PShIeP7s32RmpXPPmLHx+m5rHmESRu2UPt360gEFt6vG3Yzp4HSeqReybXESSgWeAE4FuwHki0q2C9R4CvohUlv3h8wfJSkuJiVECGtRM54lz+pC7tYA7Pl7odRxjTDXYWxzkmjdnk5aSxJPn9bHjcPsQya0zCMhR1VxV9QNvA6PKWe+vwPvA5ghmCZvPHyAjyndVhjq0QwOuPqoD787M46PZ67yOY4yJsAcnLGHRhl08emZvmmZneB0n6kWyyDUH1obcznOX/UJEmgOnAaMreyIRuUJEZojIjC1btlR50FDOrODR2+mkPNce15EBretyy4fzWb2twOs4xpgImbhoE69MW8Vlh7bluG6NvY4TEyJZ5Mrb31e2v/sTwI2qGqzsiVR1jKoOUNUBDRs2rKp85fL5A1Hf6aSslOQknjyvL0lJwj/HziVYYqcVGBNvtu4p4qb359GtaW1uPNHOhwtXJItcHhA69lQLYH2ZdQYAb4vIKuBM4FkROTWCmfapoCgYc0UOoHmdDO4e1Z0Zq3fwwtRcr+MYY6qQqvJ/H8xn994A/z6nD+kpsfcd5ZVIFrnpQEcRaSsiacC5wLjQFVS1raq2UdU2wHvAX1T1owhm2idfsbezgh+MU/s058QeTXj8y2V2WoExceT9WeuYuGgT1w/rROcmNi7l/ohYkVPVAHANTq/JxcBYVV0oIleKyJWRet2D5SuKvd2VpUSEe0/tQe2MVP7xzhyKApXuBTbGxIC8HT7uGreQQW3q8cfD2nkdJ+ZEtO+pqo5X1U6q2l5V73OXjVbV33U0UdVLVPW9SOYJh88fuy05cIb9evD0nizZuJsnv1rudRxjzEEoKVH+9e48SlR59KzeJCdF/6lN0cZOsCjD5w9EzYSpB+q4bo05Z0BLRk9ewczV272OY4w5QK9MW8UPudu47eRutKqf6XWcmGRFrowCfzCmzpOryK0nd6VZnQyuGzv3lznyjDGxI2fzbh76fAnHdGnEOQNt/sgDZUUuRCBYgj9QQlYM764sVatGKo+e1Zs12308OGGJ13GMMfshECzhn2PnkpmWzINn9IyJEZiilRW5ENE8YeqBGNyuPpcObctrP67m55W229KYWPHS9yuZm5fPXaN60KhWDa/jxDQrciFKJ0yN5Y4nZV0/rBMt62Vw4/vz2FtsvS2NiXYrtxbw2JfLOK5rY0b2slm+D5YVuRClI/nHeseTUJlpKTx4ei9Wbi3gCettaUxUKylRbnp/HmkpSdx3Wg/bTVkFrMiF8LkTpkbrrOAH6tAODThnQEtemJrL/Lx8r+MYYyrw1vQ1/LRyO7eM6Erj2rabsipYkQtR2gsx1gZoDsfNJ3WlflYa/3pvLsXBEq/jGGPKWL+zkAfGL2Fo+/rWm7IKWZELEW8dT0JlZ6Ry76k9WLJxN6MnrfA6jjEmhKpy60cLCJSU8ODpvWw3ZRWyIhciHjuehDqhexNO6tWUp77JYfmm3V7HMca4Pp6znm+WbOb6EzrbSd9VzIpciNKOJ/HYkit11yndyUxP5ob359mUPMZEgW17irjrk4X0aVmHSw9t63WcuGNFLkRpx5N4PCZXqkHNdG4/uRuz1+zkzZ/XeB3HmIR33/jF7N4b4KEzetnYlBFgRS5EQQK05ABO69ucoe3r8/DnS9i8e6/XcYxJWNNWbOWDWev485HtbAqdCLEiF6LQHyRJID0lvjdL6ZQ8RcUl3PPpYq/jGJOQigJBbv1wAa3qZfLXYzp6HSduxfe3+X5yZgVPSYieTe0a1uQvR7fnk7nrmbxsi9dxjEk4z01aQe7WAu45tQc14uzc3GhiRS6Ezx+7E6YeiKuOak+7Blnc9tECG/LLmGqUu2UPz367gpG9m3Fkp4Zex4lrVuRC+PzBuO50UlZ6SjL3ntaDNdt9PPWNDfllTHUoPScuPTWJ207u6nWcuGdFLoTPH4i7Ib32ZWj7BpzRrwVjpuTauXPGVIMPZ69j2opt3Di8i80wUA2syIVwWnKJVeQAbjmpK1npKdz84XxK7Nw5YyJmp8/PfZ8tpm+rOpw/qJXXcRKCFbkQBf5g3I52Upl6WWncfGJXpq/awXuz8ryOY0zceujzpewsLOa+U3uSZOfEVQsrciF8RYnV8STUmf1b0L91XR6asIR8X7HXcYyJO3PX7uTt6Wu4ZGgbujWr7XWchGFFLoQvQVtyAElJwt2jurPD5+fxiUu9jmNMXCkpUW7/eAENaqZz7XF2Tlx1siIXwucPJOQxuVLdm2Vz0eDWvPbjahaut3nnjKkq78xYy9y8fG4Z0ZVaNVK9jpNQ9lnkRKSTiHwtIgvc271E5NbIR6t+Bf4gGQm6u7LUdSd0pm5mGrd/vNA6oRhTBXb6/Dz8+RIGta3HqD7NvI6TcMJpyb0A/B9QDKCq84BzIxnKC4FgCf5ACVkJuruyVHZGKjee2IWZq3fwwex1XscxJuY98sVSdu0NcPeo7gkxmlK0CafIZarqz2WWBSIRxkvxPGHq/jqzXwv6tqrDgxMWk19onVCMOVDz8pzZPv4wpA1dmlhnEy+EU+S2ikh7QAFE5ExgQ0RTeaDQH98Tpu6PpCThnlE92Fbg598Tl3kdx5iY5HQ2WUj9rHSuPd46m3glnCJ3NfA80EVE1gHXAldGMpQXCoqcxmkidzwJ1aN5Nhce0ppXf1jFovW7vI5jTMx5d+Za5qzdyc0julDbOpt4Jpwip6p6HNAQ6KKqh4X5uJhSOmFqog3rVZnrT+hMncw07hi3AFXrhGJMuHb6/Dz0+VIGtqnLaX2bex0noYVTrN4HUNUCVS0d3PC9yEXyRiLMCr6/sjNTuXF4Z6av2sGH1gnFmLA99uUy8guLuXtUD+ts4rEKi5yIdBGRM4BsETk95HIJENaooiIyXESWikiOiNxUzv2jRGSeiMwRkRkictgBv5ODlCizgu+vs/q3pE/LOtw/fgm79lonFGP2ZcG6fF7/aTUXDW5N16bW2cRrlbXkOgMnA3WAkSGXfsCf9vXEIpIMPAOcCHQDzhORbmVW+xrorap9gMuAF/cvftXxFVnHk/L82gmliCe/sul4jKmMqnLHuIXUz0rjH8d38jqOASr8RlfVj4GPRWSIqv5wAM89CMhR1VwAEXkbGAUsCnmNPSHrZ+H24PSCz1pyFerZIptzB7bkf9NWcd6gVnRoVNPrSMZEpXFz1zNz9Q4ePrMX2RnW2SQahHNMbraIXC0iz4rIS6WXMB7XHFgbcjvPXfYbInKaiCwBPsNpzf2OiFzh7s6csWXLljBeev/5/HaeXGX+eUJnMtKSuefTRdYJxZhy+PwBHhi/hJ7NszmzXwuv4xhXOEXuNaAJMAyYDLQAwplds7yjrb/7dlTVD1W1C3AqcE95T6SqY1R1gKoOaNgwMlPFW8eTyjWomc7fj+3I5GVb+HbpZq/jGBN1Rk9awcZde7nzlG42jU4UCafIdVDV24ACVf0fcBLQM4zH5QEtQ263ANZXtLKqTgHai0iDMJ67yvn8AZIE0lPi7uyIKnPxkDa0a5jFPZ8uxh8o8TqOMVFj7XYfz0/JZVSfZvRvXc/rOCZEON/opV3qdopIDyAbaBPG46YDHUWkrYik4Yx3OS50BRHpIG7/WhHpB6QB28LMXqUKipxpdqy7b8XSUpK47eRurNxawCvTVnodx5io8eCEJSSJcNOJXbyOYsoIp8iNEZG6wK04RWoR8NC+HqSqAeAa4AtgMTBWVReKyJUiUjpiyhnAAhGZg9MT8xz16IBPYXHiTpi6P47u3IhjujTiP1/nsHn3Xq/jGOO5H3O38dn8DVx1VHuaZmd4HceUUWmRE5EkYJeq7lDVKaraTlUbqerz4Ty5qo5X1U6q2l5V73OXjVbV0e71h1S1u6r2UdUhqvrdQb+jA1RQFLTjcWG69aSuFAWCPPqFTa5qEluwRLnrk0U0r5PBFUe08zqOKUelRU5VS3BaY3HP5w/YkF5hatewJpce2pZ3Z+YxL2+n13GM8czb09eweMMubh7RlRr2/RGVwtldOVFErheRliJSr/QS8WTVzOcP2uDM++GaYzpQPyuNO8cttFMKTELK9xXz6BdLOaRtPUb0bOJ1HFOBcIrcZTgzEUwBZrqXGZEM5QVnVnDbXRmu2jVSuWFYF2at2cnHcyrsNGtM3Hry6+XkFxZz+8hu1mEtiu2zyKlq23IucbfzudAfIMs6nuyXM/u3oGfzbB6YsPiXqYqMSQQ5m3fz6g+rOHdQK7o3y/Y6jqmEnRTmKj2FwIQvKUm485RubNpVxHOTVngdx5hqoarc/eliMtKS+aeNTxn1rMi5fH47heBA9G9dj1F9mjFmai5rt/u8jmNMxH2zZDNTlm3h2uM6Ub9mutdxzD5YkXP5/EEyrePJAbnpxC4ki3DfZ4u9jmJMRPkDJdzz6SLaN8zi4iGtvY5jwrDP/XPuSCRl5QOr3RO+Y14gWEJRoIQs2115QJpmZ/CXo9rz2MRlTMvZytAOnozMZkzEvTJtJau2+Xjl0oGkJlsbIRaE81t6FvgRGAO8APwAvA0sE5ETIpit2viKbQaCg/WnI9rRom4Gd32yiEDQxrU08WfL7iL+83UOx3RpxFGdG3kdx4QpnCK3CujrzgLQH+gLLACOAx6OYLZqU+i3CVMPVo3UZG4Z0ZWlm3bz1s9rvI5jTJV79Iul7C0OcutJXb2OYvZDOEWui6ouLL2hqotwil5u5GJVr9Lu79aSOzjDezRhcLt6PDZxGTsK/F7HMabKzM/LZ+zMtVx6aBvaNbRJg2NJOEVuqYg8JyJHupdncXZVpvPrDAUxzSZMrRoiwh0ju7OrsJgnvlrmdRxjqoSqctcnC6mflcZfj+3odRyzn8IpcpcAOcC1wD+AXHdZMXB0hHJVK5swtep0bVqb8w9pxes/rWHpxnDm1jUmuo2bu54Zq3fwr2GdqV0j1es4Zj+FM+JJoao+pqqnqeqpqvqoqvpUtURV91RHyEgr8Du7KzOsJVclrju+M1lpydz9qY1raWKbzx/gwQlL6NG8Nmf2b7nvB5ios88iJyKHishEEVkmIrmll+oIV11KO57YKQRVo15WGv84vhPf52xj4qJNXscx5oCNnpzLhvy93DGyO8lJNj5lLApnd+V/gceBw4CBIZe4YR1Pqt6Fg1vTsVFN7v1sMUWBoNdxjNlveTt8PD95BSN7N2Ngm7ibeCVhhFPk8lV1gqpuVtVtpZeIJ6tG1vGk6qUmJ3H7yG6s2e7jpe9WeR3HmP32wIQliDgj+pjYFU6R+1ZEHhGRISLSr/QS8WTVyDqeRMbhHRtyXNfGPP3Ncjbv2ut1HGPC9mPuNj6bt4Erj2xP8zoZXscxByGcIncIMAC4H3jMvTwayVDVzecPIALpKTZMT1W79aSu+IMlPPzFUq+jGBOWYIly1yeLaF4ngz8f0d7rOOYg7bPpoqpxcZpAZXz+IFlpKTbxYQS0aZDFZYe15fnJuVw0uDW9W9bxOpIxlXp7+hoWb9jF0+f3tR7XcaDCIiciF6rq6yJyXXn3q+rjkYtVvWyancj66zEd+WDWOu78ZCEfXDXU/pkwUSvfV8yjXyxlUNt6nNSzqddxTBWobP9clvuzVgWXuOFMmGpFLlJqpqdww7DOzF6zk4/mrPM6jjEVeuLrZewsLOaOkd3sn7E4UWFLTlWfd3/eVX1xvOHz26zgkXZGvxa89uNqHpywhBO6NbFOPibqLN+0m1d/WM25A1vRvVm213FMFQnnZPCGInKziIwRkZdKL9URrrr4/AGybMLUiEpKcsa13LSriOcmrfA6jjG/oarc/ekiMtOSuf6ETl7HMVUonO6EHwPZwFfAZyGXuFHgD5JhLbmI69+6Lqf2acaYqbms3e7zOo4xv/h68WamLt/Ktcd1on7NdK/jmCoUTpHLVNUbVXWsqr5feol4smpU6A+QZcfkqsWNJ3YhWYT7xy/2OooxABQFgtz72SLaN8zi4iGtvY5jqlg4Re5TERkR8SQeKigKWlfhatI0O4Orj27PhAUbmbZiq9dxjOHl71exapuP20d2JzXZzpWNN+H8Rv+OU+gKRWSXiOwWkV2RDladCouDNjhzNbr88Ha0qJvB3Z8sIhAs8TqOSWCbd+/lqa+Xc1zXRhzZqaHXcUwEVFrkRCQJGK6qSaqaoaq1VbWWqtaupnzVoqAoQKZ1PKk2NVKTuWVEV5Zs3M3b09d6HccksIc/X4o/WMItJ3XzOoqJkEqLnKqWEGdDeJUVLFGKAiVkplpLrjoN79GEwe3q8diXS8n3xcUE8ybGzF27k/dm5nHZYW1p2yBr3w8wMSmc3ZVfisgZEqdnRvrcCVPtFILqJeKcUpBfWMwTXy/zOo5JMCUlyp2fLKRBzXSuObqD13FMBIVT5K4D3gWK9veYnIgMF5GlIpIjIjeVc/8FIjLPvUwTkd77mf+g/TrNjrXkqlvXprU5/5BWvPrDapZv2u11HJNAPp67jtlrdnLj8M7UqpHqdRwTQfsscu4xuCRVTdufY3Iikgw8A5wIdAPOE5GyO75XAkeqai/gHmDM/r+Fg2MTpnrruuM7k5WWzN2fLkJVvY5jEkBBUYAHJyyhd4tszujXwus4JsLC6i8rInVFZJCIHFF6CeNhg4AcVc1VVT/wNjAqdAVVnaaqO9ybPwLV/omzCVO9VS8rjX8c34mpy7fy9eLNXscxCeDZSTls2lXE7SO7k5QUl0dhTIhwhvW6HJgCfAHc5f68M4znbg6Edp3Lc5dV5I/AhAoyXCEiM0RkxpYtW8J46fDZ7krvXTi4NR0a1eTezxZRFAh6HcfEsTXbfLwwdSWn9W1O/9Z1vY5jqkG458kNBFa7c8v1BcKpNOX9i1Tu/igRORqnyN1Y3v2qOkZVB6jqgIYNq/ZcltKOJ3YKgXdSk5O4/eRurNrm46XvVnkdx8Sxez5bREqScOPwLl5HMdUknCK3V1X3AohIuqouATqH8bg8oGXI7RbA+rIriUgv4EVglKpuC+N5q1RpS85OBvfWEZ0ackK3xjz1zXI25Bd6HcfEoW+Xbmbiok1cc0wHmmTX8DqOqSbhFLk8EakDfARMFJGPKadYlWM60FFE2opIGnAuMC50BRFpBXwAXKSqnvQjt44n0eO2k7sRLFHu/czGtTRVqygQ5K5xC2nXIIvLD2vndRxTjfbZfFHV09yrd4rItzgzEnwexuMCInINzjG8ZOAlVV0oIle6948GbgfqA8+6p+EFVHXAAb2TA1RYbB1PokXLeplcfXQHHp+4jPMHbeXQDg28jmTixAtTclm1zcerlw0iLcXGp0wk4fauPExELlXVycAPVN6B5BeqOl5VO6lqe1W9z1022i1wqOrlqlpXVfu4l2otcOAMzgzYJJ5R4ooj2tG6fia3f7wAf8DGtTQHL2+Hj6e/zeHEHk04wsanTDjh9K68A6dDyP+5i1KB1yMZqjr5/AFEIN3+u4sKNVKTuWNkN1ZsKeDl71d6HcfEgXs+XYQg3HqyjU+ZiML5Zj8NOAUoAFDV9UCtSIaqTj6/MwNBnI5aFpOO6dKY47o25smvrROKOTiTl23hi4VOZ5PmdTK8jmM8EE6R86szFIUCiEhcjWTq8wdsLrkodMdIpxPKfdYJxRygokCQO8ctpG2DLC4/vK3XcYxHwilyY0XkeaCOiPwJ+Ap4IbKxqo/TkrMiF21a1svkqqPa8+m8DTa5qjkgL05dycqtBdx5SnfSU+xvPFGFM3blo8B7wPs458fdrqpPRTpYdSkoCtpoJ1HqyiPb07JeBrd/vJBim1zV7Id1Owt5+pschnVvbJOhJriweluo6kRV/ZeqXq+qEyMdqjr5/AE7fSBK1UhN5o6Tu5OzeY91QjH75d5PF6Eot1lnk4RXYZErnVKnnEvYU+3EAp8/SKadPhC1juvWmGO6NOLJr5azadder+OYGDB1+RYmLNjI1Ud1oEXdTK/jGI9VWORKp9Qp5xLWVDuxwucP2DG5KHfHyG4Ulyh3f7rI6ygmyu0tDnLHxwtpXT+TPx1hI5uYMHdXxrOCoqD1roxyretncfVRHfhs3gYmLbXpeEzFnpu0gtytBdwzqgc1Uu3v2liRo7A4aIMzx4Arj2pHu4ZZ3PbxAgr9Nh2P+b0VW/bw3KQVnNK7mY1sYn6R8EWuoChg0+zEgPSUZO47tSdrtxfy1DfLvY5jooyqcttHC0hPTeLWk7t6HcdEkYQucsESpShQQmaqteRiwZD29TmjXwvGTMll2abdXscxUeTD2euYtmIbNw7vQqNaNo2O+VVCF7nSCVOzrCUXM245qSu1aqRw8wfzKSkpdw5ek2B2FPi597PF9G1Vh/MHtfI6jokyCV7knGM71vEkdtTLSuP/RnRlxuodjJ2x1us4Jgo8OGEJ+YXF3H9aT5KSbAxa81tW5LBZwWPNWf1bMKhtPR6YsISte4q8jmM89PPK7bwzYy2XH9aWrk3j5swmU4USusjZrOCxSUS4/7Qe+PwBG8A5gfkDJdzy4Xya18ng78d19DqOiVIJXeRKW3I2dmXs6dCoFlcd2Z4PZ6+zc+cS1LOTcli+eQ/3nNrd/oZNhRK8yLktOet4EpOuPqYDHRrV5JYPF7DHbZWbxLB0426e+TaHUX2acUyXxl7HMVEswYucHZOLZekpyTx0Ri/W5xfy8OdLvI5jqkmwRLnhvbnUqpHKHSO7ex3HRDkrctgxuVjWv3VdLhnahld/WM3PK7d7HcdUg5e+W8ncvHzuPKU79bLSvI5jolyCFznreBIPrj+hMy3qZnDj+/PYW2xDfsWzVVsLeGziUo7r2piRvZp6HcfEgIQucgVF1vEkHmSlp/Dg6b1YubWAJ76yIb/iVUmJctMH80hNSuLeU3sgYufEmX1L6CJX6A8gAjVSE3ozxIXDOjbg7AEteGFqLvPz8r2OYyLg7elr+TF3O7ec1JUm2TZ0lwlPQn+7F/idGQjsP8L4cMtJ3aiflcYN78+jOFjidRxThTbkF/LA+MUMbV+fcwa29DqOiSEJXeR8/oAN6RVHsjNSuefUHizesIunv8nxOo6pIqrKje/PJ1CiPHh6L/un1OyXBC9yQZsVPM4M696E0/o25+lvc5i7dqfXcUwVeOOnNUxZtoWbR3ShVf1Mr+OYGJPQRa6gKGidTuLQnad0p1GtdP4xdo71toxxq7YWcN9nizm8YwMuHNza6zgmBiV0kSssDtjpA3EoOyOVR87sTe6WAh6yk8RjVrBEuW7sHFKThYfPtN2U5sAkdJErKAqSmW4tuXh0WMcGXDK0DS9/v4ppOVu9jmMOwPNTVjBrzU7uObUHTbMzvI5jYlRCFzmfP0BmqrXk4tWNw7vQrkEW1787l117i72OY/bDovW7+PfEZZzUsymn9G7mdRwTwyJa5ERkuIgsFZEcEbmpnPu7iMgPIlIkItdHMkt5fP6gDc4cxzLSknn8nD5s2l3EXeMWeR3HhKkoEOS6sXOok5nGPXbStzlIEStyIpIMPAOcCHQDzhORbmVW2w78DXg0Ujkq43PPkzPxq0/LOlx9VHven5XH5ws2eh3HhOHxictYsnE3D53R08amNActki25QUCOquaqqh94GxgVuoKqblbV6YAn+5IKiqzjSSL467Ed6dk8m5s+mMf6nYVexzGV+G75VsZMyeW8QS1tCh1TJSJZ5JoDa0Nu57nLokKwRCkKlNgpBAkgNTmJ/5zXl+JACX9/ezYBGw0lKm3ZXcS178yhQ8Oa3H6yTaFjqkYki1x5O9L1gJ5I5AoRmSEiM7Zs2XKQsRylMxBk2TG5hNC2QRb3n96T6at28OTXNohztClxTxfYvbeYp8/vZyMRmSoTySKXB4QOMtcCWH8gT6SqY1R1gKoOaNiwYZWEK3TnkrM/psQxqk9zzurfgqe/zeF7O60gqoyesoKpy7dyx8judG5Sy+s4Jo5EsshNBzqKSFsRSQPOBcZF8PX2S4HNCp6Q7hrVnXYNsrj2nTls3VPkdRwDzFy9g8e+XMZJvZpy3iAbfNlUrYgVOVUNANcAXwCLgbGqulBErhSRKwFEpImI5AHXAbeKSJ6I1I5UplAFRc7uSmvJJZbMtBSePr8f+YXFXDd2LiUlB7QH3VSRfF8xf3trNs3q1OCB03va6QKmykX0PDlVHa+qnVS1vare5y4braqj3esbVbWFqtZW1Tru9V2RzFSqsNhacomqa9Pa3H5yN6Ys28KYqblex0lYzuwC89i0ay9PndeP2jVSvY5k4lDCjnhS2pKzk8ET0wWHtGJEzyY8+sVSfsrd5nWchPTS96v4fOFGbhjemT4t63gdx8SphC1yPveYnJ0nl5hEhAfP6EWreplc/eZsNubv9TpSQvkxdxv3j1/MsO6N+dPh7byOY+JYwhc5212ZuGrXSOX5i/rj8we46o2Z+AN2/lx12JBfyDVvzqJ1/UwePau3HYczEZXARc7dXWktuYTWsXEtHjmzN7PX7OSeT218y0grCgS56vVZFPqDjLmoP7XsOJyJsAQucqW7K60ll+hO6tWUPx/Rjtd+XM3Y6Wv3/QBzQFSVO8ctZM7anTx2dm86NLLz4UzkJW6RKwogAjVSE3YTmBD/GtaZwzo04JaP5ltHlAh5+ftVvPXzWv5yVHuG92jqdRyTIBL2G77AHyQzNdmOBxgAUpKTeOb8frSsm8mVr89k9bYCryPFlW+XbubezxYxrHtjrj+hs9dxTAJJ2CLnzCVnuyrNr7IzU/nvJQMpUfjj/2bYRKtVZOnG3fz1zdl0bVqbf5/Th6Qk+8fSVJ8ELnIBsqzTiSmjbYMsRl/Yn1VbC7j6jVkU24wFB2XrniL++L/pZKQl8+IfBtgxcFPtErbIFRQFybA/OFOOIe3rc99pPZi6fCs3fzAfVRv660AUFAW47JXpbN1TxAsXD6BpdobXkUwCSthv+cJia8mZip0zsBXrd+7lya+X07BWOjcM7+J1pJjiD5Rw5eszWbh+F2Mu6m8jmhjPJGyRKygKUjvDztExFbv2uI5s3l3Es5NW0KhWOpcc2tbrSDGhpES54b25TF2+lYfP7MWxXW2Gb+OdhC1yhf4gTWrX8DqGiWIiwr2n9mDbniLu+nQR9Wqmc0rvZl7Himqqyv3jF/PRnPX8a1hnzh5gU+cYbyXuMTl/wAZnNvuUnCT857y+DGxdj+vemcMXCzd6HSmqPT5xGS9+t5JLhrbhL0e19zqOMYlb5Hz+oA3pZcJSIzWZly4dSM8W2Vzz5iy+WbLJ60hR6T9fL+epb3I4d2BLbj+5m52DaqJCAhe5gA3ObMJWMz2FVy4dRNemtbnytVlMXrbF60hR5dlJOTw+cRln9GvB/af1tHPhTNRIyCIXLFH2FpfYOTtmv2RnpPLqZYPo0KgmV7w6g2+XbvY6UlR4dlIOD3++lFF9mvHwmb2swJmokpBFzmYgMAeqTmYar19+CB0a1eRP/5vBJ3PXex3JM6rKAxMW8/DnSzmldzMeO6s3yVbgTJRJyCJXWDoDgXU8MQegXlYab10xmH6t6vK3t2fz5k9rvI5U7YIlys0fzuf5yblcOLgVT5zTh5TkhPw6MVEuIT+VBTZhqjlItWuk8r/LBnFUp4bc/OF8nvk2J2FGRtlbHORvb8/mrZ/XcvXR7blnVA/bRWmiVkIWudLdlRm2u9IchIy0ZMZcPIBT+zTjkS+W8q/35lEUCHodK6K27C7ivBd+5LN5G7h5RBf+NayL9aI0US0hmzI+a8mZKpKanMS/z+lD6/pZPPn1ctZs8zH6ov7Uy0rzOlqVW7xhF398ZTrbfX6eu6AfJ/a0OeFM9EvIllxBkbXkTNUREf5xfCf+c15f5ubtZNQz37Fo/S6vY1Wpzxds5IznphFU5b0rh1qBMzEjIYtcaceTLOt4YqrQKb2b8fYVgykqLuHUZ7/n9R9Xx/xxuqJAkDvHLeTK12fSsVFNxl1zGD2aZ3sdy5iwJWSRs44nJlL6tqrL+L8fzuB29bn1owVc8+bsmJ18ddXWAs54bhqvTFvFZYe2ZeyVQ2hs472aGJOQ3/LW8cREUoOa6bxyyUDGTM3lkS+WMmftTh44vSdHdGrodbSwlJQob/y8hgfHLyYlOYkXLh7A8d1sJgETmxKyJWcdT0ykJSUJVx7ZnrF/HkJ6ahIXv/Qz142dw44Cv9fRKpWzeQ9nP/8Dt3204JdWqRU4E8sS8lveVxRABGqkJmSNN9Wof+u6jP/b4TzzbQ7PTVrB5KVbuGF4Z87o1yKqTp7eUxRgzOQVjJ6cS0ZaMo+e1Zsz+jW30wNMzEvMIucPkpmabH/AplrUSE3mnyd05qReTbn5g/nc+P58Xpi6khuGdeb4bo09/Rz6AyW8+dNqnvomh20Ffk7p3YzbTu5Gw1rpnmUypiolZJEr8AfJTE/It2481KVJbd6/aihfLNzIw58v5YrXZtK3VR2uOLwdx3drXK0tu4KiAO/PyuOFqbms3V7IkHb1ufHELvRpWafaMhhTHRLym97nD9jgzMYTIsLwHk05rmtj3p2Zx9Pf5HDVG7NoXieDi4a05uwBLSN6IvmqrQW89uNqxk5fy+6iAL1b1uHeU3tyRMcGtmfDxKWIFjkRGQ48CSQDL6rqg2XuF/f+EYAPuERVZ0UyE5ROmJqQ9d1EiZTkJM4b1IqzB7Tkq8WbePn7lTw4YQmPfLGUwe3qMbxHU4Z1b0yjWgfXZV9VWbGlgM8XbGD8/I0s2rCLlCRhRM+mXHJoG/q1qltF78iY6BSxb3oRSQaeAY4H8oDpIjJOVReFrHYi0NG9HAI85/6MKGfCVGvJGe8lJwnDujdhWPcmLNm4i0/mrmfC/I3c9tECbvtoAe0aZNGnVR36tqpL1ya1aFong0a10kktZ9fm3uIgG/P3sn5nIfPX5TN7zU5mr93Bpl1FAPRtVYdbRnRlZO9mNMm2891MYohkc2YQkKOquQAi8jYwCggtcqOAV9UZFuJHEakjIk1VdUMEc1FQFKRWDWvJmejSpUltujSpzfUndGbZpj18tXgTs9fsZMqyLXwwa90v64lA/ax00pJ/3b3oKw6y0/fbk85b1ctkcLv69G9dl+O7NaZpdka1vRdjokUkv+mbA2tDbufx+1Zaees0B35T5ETkCuAKgFatWh10sMHt6lM7w4qciU4iQucmtejcpBbg7HLM21HIii172Ji/lw35e9m8ey+B4K9DhqWnJtE0O4MmtWvQNLsGnZvUon5N6yFpTCS/6cs7il12IL9w1kFVxwBjAAYMGHDQgwHedGKXg30KY6qNiNCyXiYt62V6HcWYmBPJPst5QMuQ2y2A9QewjjHGGHNAIlnkpgMdRaStiKQB5wLjyqwzDrhYHIOB/EgfjzPGGJM4Ira7UlUDInIN8AXOKQQvqepCEbnSvX80MB7n9IEcnFMILo1UHmOMMYknor0vVHU8TiELXTY65LoCV0cygzHGmMQVPSPEGmOMMVXMipwxxpi4ZUXOGGNM3LIiZ4wxJm6J0/cjdojIFmB1FTxVA2BrFTxPdYq1zLGWF2Ivc6zlBctcHWItLxx85taq2rDswpgrclVFRGao6gCvc+yPWMsca3kh9jLHWl6wzNUh1vJC5DLb7kpjjDFxy4qcMcaYuJXIRW6M1wEOQKxljrW8EHuZYy0vWObqEGt5IUKZE/aYnDHGmPiXyC05Y4wxcc6KnDHGmLiVcEVORIaLyFIRyRGRm7zOUx4RaSki34rIYhFZKCJ/d5ffKSLrRGSOexnhddZQIrJKROa72Wa4y+qJyEQRWe7+rOt1TgAR6RyyHeeIyC4RuTbatrGIvCQim0VkQciyCrepiPyf+9leKiLDoijzIyKyRETmiciHIlLHXd5GRApDtvfoCp+4evNW+DmI4m38TkjeVSIyx10eDdu4ou+0yH+WVTVhLjhT/qwA2gFpwFygm9e5ysnZFOjnXq8FLAO6AXcC13udr5Lcq4AGZZY9DNzkXr8JeMjrnBV8LjYCraNtGwNHAP2ABfvapu5nZC6QDrR1P+vJUZL5BCDFvf5QSOY2oetF0TYu93MQzdu4zP2PAbdH0Tau6Dst4p/lRGvJDQJyVDVXVf3A28AojzP9jqpuUNVZ7vXdwGKgubepDtgo4H/u9f8Bp3oXpULHAitUtSpG0qlSqjoF2F5mcUXbdBTwtqoWqepKnHkaB1VHzlDlZVbVL1U14N78EWhR3bkqUsE2rkjUbuNSIiLA2cBb1RqqEpV8p0X8s5xoRa45sDbkdh5RXjxEpA3QF/jJXXSNu8vnpWjZ9RdCgS9FZKaIXOEua6zubO/uz0aepavYufz2CyGatzFUvE1j5fN9GTAh5HZbEZktIpNF5HCvQpWjvM9BLGzjw4FNqro8ZFnUbOMy32kR/ywnWpGTcpZF7TkUIlITeB+4VlV3Ac8B7YE+wAacXRLR5FBV7QecCFwtIkd4HWhfRCQNOAV4110U7du4MlH/+RaRW4AA8Ia7aAPQSlX7AtcBb4pIba/yhajocxD12xg4j9/+0xY127ic77QKVy1n2QFt50QrcnlAy5DbLYD1HmWplIik4nwY3lDVDwBUdZOqBlW1BHgBD3aTVEZV17s/NwMf4uTbJCJNAdyfm71LWK4TgVmqugmifxu7KtqmUf35FpE/ACcDF6h74MXdHbXNvT4T59hLJ+9SOir5HET7Nk4BTgfeKV0WLdu4vO80quGznGhFbjrQUUTauv/BnwuM8zjT77j71P8LLFbVx0OWNw1Z7TRgQdnHekVEskSkVul1nI4GC3C27x/c1f4AfOxNwgr95r/eaN7GISrapuOAc0UkXUTaAh2Bnz3I9zsiMhy4EThFVX0hyxuKSLJ7vR1O5lxvUv6qks9B1G5j13HAElXNK10QDdu4ou80quOz7GWPGy8uwAicnj0rgFu8zlNBxsNwmubzgDnuZQTwGjDfXT4OaOp11pDM7XB6Q80FFpZuW6A+8DWw3P1Zz+usIZkzgW1AdsiyqNrGOAV4A1CM89/tHyvbpsAt7md7KXBiFGXOwTnGUvp5Hu2ue4b7eZkLzAJGRkneCj8H0bqN3eWvAFeWWTcatnFF32kR/yzbsF7GGGPiVqLtrjTGGJNArMgZY4yJW1bkjDHGxC0rcsYYY+KWFTljjDFxy4qcMVFMnJkRMr3OYUysslMIjIliIrIKGKCqW73OYkwsspacMVHCHTXmMxGZKyILROQOoBnwrYh8665zgoj8ICKzRORddyzA0rn8HhKRn91LB3f5We5zzRWRKd69O2O8YUXOmOgxHFivqr1VtQfwBM54fUer6tEi0gC4FThOnYGwZ+AMuFtql6oOAp52HwtwOzBMVXvjDERtTEKxImdM9JgPHOe2yA5X1fwy9w/GmUzye3fW5z/gTPRa6q2Qn0Pc698Dr4jIn3AmhzUmoaR4HcAY41DVZSLSH2dMvwdE5MsyqwgwUVXPq+gpyl5X1StF5BDgJGCOiPRRd0R6YxKBteSMiRIi0gzwqerrwKNAP2A3UMtd5Ufg0JDjbZkiEjplyjkhP39w12mvqj+p6u3AVn47fYkxcc9acsZEj57AIyJSgjO6/FU4ux0niMgG97jcJcBbIpLuPuZWnFk1ANJF5Cecf15LW3uPiEhHnFbg1zgj0RuTMOwUAmPigJ1qYEz5bHelMcaYuGUtOWOMMXHLWnLGGGPilhU5Y4wxccuKnDHGmLhlRc4YY0zcsiJnjDEmbv0/PZDGt4sF9o4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.nn.Parameter(torch.zeros(0))\n",
    "opt = torch.optim.SGD([p], lr=5)\n",
    "multiplier = WarmupParamScheduler(\n",
    "    CosineParamScheduler(0.1, 0.0001),\n",
    "    warmup_factor = 0.001,\n",
    "    warmup_length = 0.05,\n",
    "    warmup_method = 'linear'\n",
    ")\n",
    "total = 100\n",
    "scheduler = LRMultiplier(opt, multiplier, total)\n",
    "steps, lrs = [], []\n",
    "\n",
    "for _iter in range(total * 2):\n",
    "    p.sum().backward()\n",
    "    opt.step()\n",
    "    lrs.append(opt.param_groups[0][\"lr\"])\n",
    "    steps.append(_iter)\n",
    "\n",
    "    scheduler.step()\n",
    "draw(steps, lrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
